from pymongo import MongoClient
from bs4 import BeautifulSoup
from pprint import pprint
import requests
import datetime
import json
import time
import re

client = MongoClient('47.104.130.19', 27017)
db = client['knx_posts_db']
offical_posts_coll = db['offical_posts_coll']


class POST():
    def __init__(self):
        self.scrapy()

    def scrapy(self):
        url = 'https://recruit.csvw.com/recruitment/index/campus/aid/10068/cid/1'
        r = requests.get(url)

        for branch in BeautifulSoup(r.text, 'lxml').select('.com-branch a'):
            url = 'https://recruit.csvw.com' + branch.get('href')
            soup = BeautifulSoup(requests.get(url).text, 'lxml')
            location = re.findall('工作地点：.+', soup.find(text = re.compile('工作地点：')).parent.get_text())[0].split('：')[1]

            for item in soup.select('.joblist tbody tr'):
                name = item.select('td')[1].get_text().strip()
                description = item.select('td')[2].get_text().strip()
                major = item.select('td')[3].get_text().strip()

                item = {
                    "url": url,
                    'edu': '',
                    'exp': [],
                    'name': name,
                    'date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
                    'lang': '',
                    'place': '',
                    'major': major,
                    'count': '',
                    'salary': [],
                    'toSchool': True,
                    'welfare': [],
                    'funType': '',
                    'company': '上汽集团',
                    'location': location,
                    'industry': '汽车制造',
                    'keywords': [],
                    'platform': 'offical',
                    'searchKeyword': '',
                    'description': description,
                    'subIndustry': '',
                    'stime': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                }

                result = offical_posts_coll.insert_one(item)

                if result.inserted_id:
                    print(result.inserted_id)


POST()
